{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading original dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "real= pd.read_csv('../gr_id.csv',sep='\\t') \n",
    "fake= pd.read_csv('../gf_id.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((15931, 15), (4567, 15))"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "real.shape, fake.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "real['label']=1\n",
    "fake['label']=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20498, 15)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset=pd.concat([real,fake])\n",
    "dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'url', 'text', 'images', 'top_img', 'keywords', 'authors',\n",
       "       'canonical_link', 'title', 'meta_data', 'movies', 'publish_date',\n",
       "       'source', 'summary', 'label'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20498, 15)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>url</th>\n",
       "      <th>text</th>\n",
       "      <th>images</th>\n",
       "      <th>top_img</th>\n",
       "      <th>keywords</th>\n",
       "      <th>authors</th>\n",
       "      <th>canonical_link</th>\n",
       "      <th>title</th>\n",
       "      <th>meta_data</th>\n",
       "      <th>movies</th>\n",
       "      <th>publish_date</th>\n",
       "      <th>source</th>\n",
       "      <th>summary</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>https://www.brides.com/story/teen-mom-jenelle-...</td>\n",
       "      <td>When you get married on national TV (and who i...</td>\n",
       "      <td>['https://media.brides.com/photos/59ca68fc3d68...</td>\n",
       "      <td>https://media.brides.com/photos/59ca68fc3d6836...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>https://www.brides.com/story/teen-mom-jenelle-...</td>\n",
       "      <td>Teen Mom Star Jenelle Evans' Wedding Dress Is ...</td>\n",
       "      <td>{'msapplication-tap-highlight': 'no', 'viewpor...</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>https://www.brides.com</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>https://www.dailymail.co.uk/tvshowbiz/article-...</td>\n",
       "      <td>Kylie Jenner reportedly doesn't want to talk a...</td>\n",
       "      <td>['https://i.dailymail.co.uk/1s/2019/04/30/22/1...</td>\n",
       "      <td>https://i.dailymail.co.uk/i/pix/2017/07/05/21/...</td>\n",
       "      <td>[]</td>\n",
       "      <td>['Ross Mcdonagh For Dailymail.Com']</td>\n",
       "      <td>https://www.dailymail.co.uk/tvshowbiz/article-...</td>\n",
       "      <td>Kylie Jenner refusing to discuss Tyga on Life ...</td>\n",
       "      <td>{'keywords': 'dailymail,TV&amp;Showbiz,Tyga,Kylie ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1.499290e+09</td>\n",
       "      <td>https://www.dailymail.co.uk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                                url  \\\n",
       "0           0  https://www.brides.com/story/teen-mom-jenelle-...   \n",
       "1           0  https://www.dailymail.co.uk/tvshowbiz/article-...   \n",
       "\n",
       "                                                text  \\\n",
       "0  When you get married on national TV (and who i...   \n",
       "1  Kylie Jenner reportedly doesn't want to talk a...   \n",
       "\n",
       "                                              images  \\\n",
       "0  ['https://media.brides.com/photos/59ca68fc3d68...   \n",
       "1  ['https://i.dailymail.co.uk/1s/2019/04/30/22/1...   \n",
       "\n",
       "                                             top_img keywords  \\\n",
       "0  https://media.brides.com/photos/59ca68fc3d6836...       []   \n",
       "1  https://i.dailymail.co.uk/i/pix/2017/07/05/21/...       []   \n",
       "\n",
       "                               authors  \\\n",
       "0                                   []   \n",
       "1  ['Ross Mcdonagh For Dailymail.Com']   \n",
       "\n",
       "                                      canonical_link  \\\n",
       "0  https://www.brides.com/story/teen-mom-jenelle-...   \n",
       "1  https://www.dailymail.co.uk/tvshowbiz/article-...   \n",
       "\n",
       "                                               title  \\\n",
       "0  Teen Mom Star Jenelle Evans' Wedding Dress Is ...   \n",
       "1  Kylie Jenner refusing to discuss Tyga on Life ...   \n",
       "\n",
       "                                           meta_data movies  publish_date  \\\n",
       "0  {'msapplication-tap-highlight': 'no', 'viewpor...     []           NaN   \n",
       "1  {'keywords': 'dailymail,TV&Showbiz,Tyga,Kylie ...     []  1.499290e+09   \n",
       "\n",
       "                        source  summary  label  \n",
       "0       https://www.brides.com      NaN      1  \n",
       "1  https://www.dailymail.co.uk      NaN      1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### cleaning dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "with open('gossip/demo_real.txt', 'r') as data:\n",
    "    d = ast.literal_eval(data.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "poli_top=pd.DataFrame(list(d.items()), columns=['url', 'top_img'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15071, 2)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "poli_top.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('gossip/demo_fake.txt', 'r') as data:\n",
    "    d = ast.literal_eval(data.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "poli_fake=pd.DataFrame(list(d.items()), columns=['url', 'top_img'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4204, 2)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "poli_fake.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "poli_image=pd.concat([poli_top,poli_fake])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "tempDataset = dataset.values.tolist()\n",
    "tempPoli = poli_image.values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20498, 19275)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tempDataset), len(tempPoli)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=(dataset['title']+ ' '+ dataset['text']).tolist()\n",
    "label=dataset['label'].tolist()\n",
    "links = dataset['url'].tolist()\n",
    "labelLinks = [(label[i], links[i]) for i in range(len(label))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20498"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(labelLinks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, test_data, train_label,test_label = train_test_split(data, labelLinks,test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"train_embeddings = np.array()\\ntest_embeddings = np.load('poli_embed_test.npy')\""
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"train_embeddings = np.array()\n",
    "test_embeddings = np.load('poli_embed_test.npy')\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20498"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_data) + len(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"trainEmbeddingJson = {}\n",
    "testEmbeddingJson = {}\n",
    "for i in range(len(train_embeddings)):\n",
    "    trainEmbeddingJson[train_data[i]] = (train_embeddings[i], train_label[i])\n",
    "for i in range(len(test_embeddings)):\n",
    "    testEmbeddingJson[test_data[i]] = (test_embeddings[i], test_label[i])\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "imageDict = {}\n",
    "for i in tempPoli:\n",
    "    imageDict[i[0]] = i[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19275"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(imageDict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainCounter = 0\n",
    "testCounter = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1,\n",
       " 'https://www.theguardian.com/music/2018/mar/27/prince-had-exceedingly-high-level-of-fentanyl-in-body-when-he-died')"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_label[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'for i in testEmbeddingJson_withImage:\\n    print(testEmbeddingJson_withImage[i])\\n    break'"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"for i in testEmbeddingJson_withImage:\n",
    "    print(testEmbeddingJson_withImage[i])\n",
    "    break\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainEmbeddingJson_withImage = {}\n",
    "for i in range(len(train_data)):\n",
    "    try:\n",
    "        if imageDict[train_label[i][1]] != {}:\n",
    "            if os.path.isfile('/home/karmanya/finalExperiment/gossip/merged/' + imageDict[train_label[i][1]]['Top_img'] + '.jpg'):\n",
    "                trainCounter += 1\n",
    "                try:\n",
    "                    shutil.copy('/home/karmanya/finalExperiment/gossip/merged/' + imageDict[train_label[i][1]]['Top_img'] + '.jpg', '/home/karmanya/finalExperiment/imageDataset/gossip/train/' + imageDict[train_label[i][1]]['Top_img'] + '.jpg')\n",
    "                except Exception as f:\n",
    "                    print(f)\n",
    "                trainEmbeddingJson_withImage[train_data[i]] = [train_label[i][0], train_label[i][1], imageDict[train_label[i][1]]]\n",
    "    except Exception as e:\n",
    "        #print(e)\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "testEmbeddingJson_withImage = {}\n",
    "for i in range(len(test_data)):\n",
    "    try:\n",
    "        if imageDict[test_label[i][1]] != {}:\n",
    "            if os.path.isfile('/home/karmanya/finalExperiment/gossip/merged/' + imageDict[test_label[i][1]]['Top_img'] + '.jpg'):\n",
    "                testCounter += 1\n",
    "                shutil.copy('/home/karmanya/finalExperiment/gossip/merged/' + imageDict[test_label[i][1]]['Top_img'] + '.jpg', '/home/karmanya/finalExperiment/imageDataset/gossip/test/' + imageDict[test_label[i][1]]['Top_img'] + '.jpg')\n",
    "                testEmbeddingJson_withImage[test_data[i]] = [test_label[i][0], test_label[i][1], imageDict[test_label[i][1]]]\n",
    "    except Exception as e:\n",
    "        #print(e)\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15632, 3894)"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainCounter, testCounter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in trainEmbeddingJson_withImage:\n",
    "    temp = trainEmbeddingJson_withImage[i]\n",
    "    trainEmbeddingJson_withImage[i] = [temp[1][0], temp[1][1], imageDict[temp[1][1]]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in testEmbeddingJson_withImage:\n",
    "    temp = testEmbeddingJson_withImage[i]\n",
    "    testEmbeddingJson_withImage[i] = [temp[1][0], temp[1][1], imageDict[temp[1][1]]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('imageDataset/gossip/trainJson.json', 'w') as f:\n",
    "    json.dump(trainEmbeddingJson_withImage, f)\n",
    "with open('imageDataset/gossip/testJson.json', 'w') as f:\n",
    "    json.dump(testEmbeddingJson_withImage, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('imageDataset/gossip/imageLinkMapping.json', 'w') as f:\n",
    "    json.dump(imageDict, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'duplicates = []\\nfor i in tempDataset:\\n    if i[1] in duplicates:\\n        print(i[1])\\n    else:\\n        duplicates.append(i[1])'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"duplicates = []\n",
    "for i in tempDataset:\n",
    "    if i[1] in duplicates:\n",
    "        print(i[1])\n",
    "    else:\n",
    "        duplicates.append(i[1])\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in tempDataset:\n",
    "    for j in tempPoli:\n",
    "        if i[1] == j[0]:\n",
    "            updatedDataset.append(i + [j[1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [],
   "source": [
    "#updatedDataset = pd.DataFrame(updatedDataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [],
   "source": [
    "#updatedDataset.columns = newFields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(661, 679)"
      ]
     },
     "execution_count": 357,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tempPoli), len(updatedDataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "744"
      ]
     },
     "execution_count": 360,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_data) + len(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "744"
      ]
     },
     "execution_count": 361,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_embeddings) + len(test_embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "574"
      ]
     },
     "execution_count": 362,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trainEmbeddingJson) + len(testEmbeddingJson)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainEmbeddingJson = {}\n",
    "testEmbeddingJson = {}\n",
    "for i in range(len(train_embeddings)):\n",
    "    trainEmbeddingJson[train_data[i]] = (train_embeddings[i], train_label[i])\n",
    "for i in range(len(test_embeddings)):\n",
    "    testEmbeddingJson[test_data[i]] = (test_embeddings[i], test_label[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainCounter = 0\n",
    "testCounter = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(679, 574)"
      ]
     },
     "execution_count": 289,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(updatedDataset), len(trainEmbeddingJson) + len(testEmbeddingJson)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in updatedDataset:\n",
    "    if i[-1] != {}:\n",
    "        temp = i[-8]+ ' '+ i[2]\n",
    "        if temp in trainEmbeddingJson:\n",
    "            trainCounter += 1\n",
    "        elif temp in testEmbeddingJson:\n",
    "            testCounter += 1\n",
    "    elif i[-1] == {}:\n",
    "        temp = i[-8]+ ' '+ i[2]\n",
    "        if temp in trainEmbeddingJson:\n",
    "            del trainEmbeddingJson[temp]\n",
    "        elif temp in testEmbeddingJson:\n",
    "            del testEmbeddingJson[temp]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(452, 110)"
      ]
     },
     "execution_count": 284,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainCounter, testCounter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(455, 119)"
      ]
     },
     "execution_count": 285,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trainEmbeddingJson), len(testEmbeddingJson)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = []\n",
    "with open('training_sentences.txt', 'r') as f:\n",
    "    lines = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'MITT ROMNEY: ‘TRUMP HAS SOME VERY SERIOUS EMOTIONAL PROBLEMS, HE MUST RESIGN BEFORE WE…’ AMID AN INTERVIEW ON CNN’S “AC360,” FORMER PRESIDENTIAL CANDIDATE MITT ROMNEY URGED PRESIDENT TRUMP TO LEAVE OFFICE OVER THE SEXUAL MISCONDUCT CLAIMS AGAINST HIM AND COMMENTED THAT THE PRESIDENT “HAS SOME VERY SERIOUS EMOTIONAL PROBLEMS.”\\n'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "URL_poli=dataset['url'].tolist()\n",
    "TITLE_poli=dataset['title'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "path='/media/data_dump_2/Shivangi/google_drive_images/Images/politifact/image_ids_to_articles_poli_top_real.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "with open('demo.txt', 'r') as data:\n",
    "    d= ast.literal_eval(data.read())       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "poli_top=pd.DataFrame(list(d.items()), columns=['url', 'top_img'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>top_img</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://www.nfib-sbet.org/</td>\n",
       "      <td>{'Top_img': 'TNvleUERVcteFhXLFMOhJNNKtVJ7Gvtj'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://www.cq.com/doc/newsmakertranscripts-494...</td>\n",
       "      <td>{'Top_img': 'F5EvpyGbVWnmGxY51SFwiwFApug6uCQD'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://web.archive.org/web/20110811143753/htt...</td>\n",
       "      <td>{'Top_img': 'DYJ719DYWZuuhTiYC24sKe8j4xtXAmSS'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://web.archive.org/web/20070820164107/htt...</td>\n",
       "      <td>{'Top_img': 'nhnAUdrFS91vNVbtAC9GJHYkCcJRJEWl'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://www.politifact.com/truth-o-meter/statem...</td>\n",
       "      <td>{'Top_img': 'B8gcMTZ9WskrcKCyVXtLr1df9gUbArSB'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>https://www.law.cornell.edu/constitution/amend...</td>\n",
       "      <td>{'Top_img': 'wV4ZL7x32sUA3ywBfOj4bqt1FK7modqs'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>https://web.archive.org/web/20090913221204/htt...</td>\n",
       "      <td>{'Top_img': 'y7ZQMDtegVuKlqLfU0fDRjzaeVOT9mf8'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>https://web.archive.org/web/20080623122709/htt...</td>\n",
       "      <td>{'Top_img': 'GBv7hlj158k63w8HmPEJRuESLwqxgFrv'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>http://www.cbsnews.com/blogs/2010/02/23/politi...</td>\n",
       "      <td>{'Top_img': '9HDdEGQ17SAzBEOlvzDnrzd8OdtzczjB'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>http://data.bls.gov/timeseries/CES0500000001?o...</td>\n",
       "      <td>{'Top_img': 'SQ3WBpBmtpwTv1xWkaAwHIq53s6mYnUP'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>http://www.kff.org/healthreform/upload/8061.pdf</td>\n",
       "      <td>{'Top_img': 'qLXWHatMmE4ewhKbklBbknHS78Dp55kW'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>http://abcnews.go.com/ThisWeek/week-transcript...</td>\n",
       "      <td>{'Top_img': 'NCi9wPcK0ANwMx4xBTFnjMiKDAxElLbk'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>https://youtu.be/EiZW41bVF9I</td>\n",
       "      <td>{'Top_img': 'W00QKgIsasu1zh6U7d2YHPAYxQgZIZ2w'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>https://www.youtube.com/watch?v=AtJWEAj1vZM</td>\n",
       "      <td>{'Top_img': 'U8CdJ7HvSZQPQoJKp80RkzScl2Gch3rW'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>http://www.politifact.com/truth-o-meter/statem...</td>\n",
       "      <td>{'Top_img': 'VHcOJvN9ja63B7Q5Kmw7jTx8u4Em4Ppi'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>http://www.epi.org/blog/lost-decade-working-ag...</td>\n",
       "      <td>{'Top_img': 'iKZ9ap7dK36FGarboQzqLsE9PA4c5dmu'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>http://www.senate.gov/pagelayout/reference/nom...</td>\n",
       "      <td>{'Top_img': '3InzvZMnmLXAWFnX4cLmaZIxQKDpploa'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>https://medium.com/@PolitiFact/politifacts-ann...</td>\n",
       "      <td>{'Top_img': 'EB78w48u8ncBBruU8fjHG5kPu13v0fnc'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>https://web.archive.org/web/20100523071827/htt...</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>http://abcnews.go.com/ThisWeek/week-transcript...</td>\n",
       "      <td>{'Top_img': '5QktBFq3fpnIjSje5Jd0KlcTObBw6kKM'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>https://web.archive.org/web/20161007194921/htt...</td>\n",
       "      <td>{'Top_img': 'MifJVwUOxoXl1wFgiXMdmlE6FRFtz556'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>http://www.senate.gov/legislative/LIS/roll_cal...</td>\n",
       "      <td>{'Top_img': 'oGAiJSDBU9SkvMpqnlruS6tBpbKTmC1Q'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>http://www.youtube.com/watch?v=wyoZJUKL3wA&amp;fea...</td>\n",
       "      <td>{'Top_img': 'fWH0R2f8ok8mzjMfvNRE6yQuTwnHyO8f'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>http://www.bls.gov/ces/</td>\n",
       "      <td>{'Top_img': 'vK7Vu5soPM14EXQFWfChkcMYw50fcMf8'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>http://www.nytimes.com/2011/07/11/opinion/11kr...</td>\n",
       "      <td>{'Top_img': 'L6rrNgReN4Jv0HCNimFw2SmUdgVp1w44'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>http://www.nytimes.com/2008/06/03/us/politics/...</td>\n",
       "      <td>{'Top_img': 'HZrLmuOmu3vGOEAaYY16I9nU4usE2e5c'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>http://www.politico.com/arena/perm/John_Lewis_...</td>\n",
       "      <td>{'Top_img': 'zZRnNcFnG8mE98pdximr9XH3vuoBZfNT'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>https://web.archive.org/web/20091229194623/htt...</td>\n",
       "      <td>{'Top_img': '6VLIS8SI9oAKlwOaY3D8vNbE2JMSBh7R'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>https://medium.com/hillary-for-america/to-youn...</td>\n",
       "      <td>{'Top_img': 'dBzWyRQmjkC5fLXVkU5puIVEvWsWZE2H'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>https://apnews.com/c810d7de280a47e88848b0ac746...</td>\n",
       "      <td>{'Top_img': 'X5QQEWWU42DZ4z6BIYuHi0ZLAmnCYiTY'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <td>http://www.press.org/news-multimedia/videos/np...</td>\n",
       "      <td>{'Top_img': 'UkhSiFbFRW7HHGncbZV4bpHumSUnYJzM'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>http://www.cbsnews.com/8301-3460_162-57500549/...</td>\n",
       "      <td>{'Top_img': 'DtTdzqOZ0zSZ8NUsvTFWvaIJC8NOvZyj'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>https://web.archive.org/web/20120619023435/htt...</td>\n",
       "      <td>{'Top_img': 'xQm9QeN9olDeAll4cdOdOBLOxwgLTES8'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>http://transcripts.cnn.com/TRANSCRIPTS/1207/08...</td>\n",
       "      <td>{'Top_img': 'qts3oDLkZ8K1AFrAzWGcmdgwkVQlADvG'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>https://web.archive.org/web/20080619025707/htt...</td>\n",
       "      <td>{'Top_img': 'eSGHW8cCpuCeZCy4CfQxPG93UMwbF7Vj'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365</th>\n",
       "      <td>http://abcnews.go.com/ThisWeek/week-transcript...</td>\n",
       "      <td>{'Top_img': '6BsVhYlDnZXzzKULjNEhianebIGxuOPE'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>366</th>\n",
       "      <td>http://www.nytimes.com/2009/02/09/us/politics/...</td>\n",
       "      <td>{'Top_img': 'EbFJqhoABxdaDJ7dI8PAYEeQVBtS9F6o'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367</th>\n",
       "      <td>https://web.archive.org/web/20090120134739/htt...</td>\n",
       "      <td>{'Top_img': 'kTsxjig8XvbrHhkjBbpmZKS0efizQYLD'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>368</th>\n",
       "      <td>https://web.archive.org/web/20090910073834/htt...</td>\n",
       "      <td>{'Top_img': 'jFZOJrrMiRgooCJWv1YjxLT3ly8iIByI'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369</th>\n",
       "      <td>https://web.archive.org/web/20090919163055/htt...</td>\n",
       "      <td>{'Top_img': 'x9HCdWe79x27h181pAdTmTlf1Z5DTQxU'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>370</th>\n",
       "      <td>http://transcripts.cnn.com/TRANSCRIPTS/0804/13...</td>\n",
       "      <td>{'Top_img': 'BUekE6CUNKbTK8mt4I2JJFKqjxfA6i4E'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371</th>\n",
       "      <td>https://web.archive.org/web/20070808173432/htt...</td>\n",
       "      <td>{'Top_img': 'h6G3SGo4Q7E4rX7bC4eQarBaoWX50imd'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>372</th>\n",
       "      <td>http://thomas.loc.gov/cgi-bin/query/D?c108:5:....</td>\n",
       "      <td>{'Top_img': 'Rw7GCoj29yd6OINmpopXWFVTOXWmvIIL'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>373</th>\n",
       "      <td>http://abcnews.go.com/ThisWeek/week-transcript...</td>\n",
       "      <td>{'Top_img': '0fNmYlMmlJgQlEMXCp10RIqo07gmqicM'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>374</th>\n",
       "      <td>http://www.colbertnation.com/the-colbert-repor...</td>\n",
       "      <td>{'Top_img': 'An8cnTMZNx4ju86WZeDwcvnXt6bSNrOA'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>375</th>\n",
       "      <td>http://www.politifact.com/truth-o-meter/statem...</td>\n",
       "      <td>{'Top_img': 'BONw3V1r0luiQ6bvp3knSzY4q1BNaqlT'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>376</th>\n",
       "      <td>https://web.archive.org/web/20150816005009/htt...</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377</th>\n",
       "      <td>http://www.rawstory.com/rs/2012/07/17/sen-sand...</td>\n",
       "      <td>{'Top_img': 'JCiDofIqViLZSUQA9I0L0AdivnlwmjfJ'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>378</th>\n",
       "      <td>http://www.politicsdaily.com/2011/02/06/transc...</td>\n",
       "      <td>{'Top_img': '86VQIQFxqGFRVJjCifFMewDQVNteu0UO'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379</th>\n",
       "      <td>http://www.nytimes.com/2008/02/26/us/politics/...</td>\n",
       "      <td>{'Top_img': 'a0x9A12cmC8JWAXDRrTPs86xShCEU3vc'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>380</th>\n",
       "      <td>http://www.politifact.com/texas/statements/201...</td>\n",
       "      <td>{'Top_img': 'qy8BQvHFDSgfl0dqgLur8XaewbNWJte9'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>381</th>\n",
       "      <td>https://www.washingtonpost.com/news/the-fix/wp...</td>\n",
       "      <td>{'Top_img': 'm8P3orzJ06kfF0jg2HmjfnF3jMSb8xjh'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382</th>\n",
       "      <td>http://www.cq.com/doc/newsmakertranscripts-395...</td>\n",
       "      <td>{'Top_img': 'WfwLXBWL3kESRdHB9ksEilmnZMcOUZtJ'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383</th>\n",
       "      <td>http://www.thedailyshow.com/full-episodes/mon-...</td>\n",
       "      <td>{'Top_img': 'JevSKniuEt66nIoKszr2VC89JlRfE7MJ'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384</th>\n",
       "      <td>http://select.nytimes.com/gst/abstract.html?re...</td>\n",
       "      <td>{'Top_img': 'XIami3akSNyJzijgc6tf0HkMhmZmT7AT'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>385</th>\n",
       "      <td>http://www.politifact.com/truth-o-meter/promis...</td>\n",
       "      <td>{'Top_img': 'tjg7cSY2CUJ6NtDw3CfddbxSbiznjahC'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>http://www.cnn.com/2017/01/05/politics/border-...</td>\n",
       "      <td>{'Top_img': 'tAKcEzennyCleZiMbqBDhRMGgNQUIjLy'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>387</th>\n",
       "      <td>http://www.youtube.com/watch?v=4O8CxZ1OD58</td>\n",
       "      <td>{'Top_img': 'vgTBvxWraHslIeqmQybPUhLPikP1rzp2'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>388</th>\n",
       "      <td>http://www.youtube.com/watch?v=EhyMplwY6HY</td>\n",
       "      <td>{'Top_img': '5W20wJ2n3eWxudEijJ5eCopF78lbAYIg'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>389</th>\n",
       "      <td>https://web.archive.org/web/20071102131244/htt...</td>\n",
       "      <td>{'Top_img': 'Zy1yA9joK5UgSybPCJ2m6stszAq7NQvF'}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>390 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   url  \\\n",
       "0                            http://www.nfib-sbet.org/   \n",
       "1    http://www.cq.com/doc/newsmakertranscripts-494...   \n",
       "2    https://web.archive.org/web/20110811143753/htt...   \n",
       "3    https://web.archive.org/web/20070820164107/htt...   \n",
       "4    http://www.politifact.com/truth-o-meter/statem...   \n",
       "5    https://www.law.cornell.edu/constitution/amend...   \n",
       "6    https://web.archive.org/web/20090913221204/htt...   \n",
       "7    https://web.archive.org/web/20080623122709/htt...   \n",
       "8    http://www.cbsnews.com/blogs/2010/02/23/politi...   \n",
       "9    http://data.bls.gov/timeseries/CES0500000001?o...   \n",
       "10     http://www.kff.org/healthreform/upload/8061.pdf   \n",
       "11   http://abcnews.go.com/ThisWeek/week-transcript...   \n",
       "12                        https://youtu.be/EiZW41bVF9I   \n",
       "13         https://www.youtube.com/watch?v=AtJWEAj1vZM   \n",
       "14   http://www.politifact.com/truth-o-meter/statem...   \n",
       "15   http://www.epi.org/blog/lost-decade-working-ag...   \n",
       "16   http://www.senate.gov/pagelayout/reference/nom...   \n",
       "17   https://medium.com/@PolitiFact/politifacts-ann...   \n",
       "18   https://web.archive.org/web/20100523071827/htt...   \n",
       "19   http://abcnews.go.com/ThisWeek/week-transcript...   \n",
       "20   https://web.archive.org/web/20161007194921/htt...   \n",
       "21   http://www.senate.gov/legislative/LIS/roll_cal...   \n",
       "22   http://www.youtube.com/watch?v=wyoZJUKL3wA&fea...   \n",
       "23                             http://www.bls.gov/ces/   \n",
       "24   http://www.nytimes.com/2011/07/11/opinion/11kr...   \n",
       "25   http://www.nytimes.com/2008/06/03/us/politics/...   \n",
       "26   http://www.politico.com/arena/perm/John_Lewis_...   \n",
       "27   https://web.archive.org/web/20091229194623/htt...   \n",
       "28   https://medium.com/hillary-for-america/to-youn...   \n",
       "29   https://apnews.com/c810d7de280a47e88848b0ac746...   \n",
       "..                                                 ...   \n",
       "360  http://www.press.org/news-multimedia/videos/np...   \n",
       "361  http://www.cbsnews.com/8301-3460_162-57500549/...   \n",
       "362  https://web.archive.org/web/20120619023435/htt...   \n",
       "363  http://transcripts.cnn.com/TRANSCRIPTS/1207/08...   \n",
       "364  https://web.archive.org/web/20080619025707/htt...   \n",
       "365  http://abcnews.go.com/ThisWeek/week-transcript...   \n",
       "366  http://www.nytimes.com/2009/02/09/us/politics/...   \n",
       "367  https://web.archive.org/web/20090120134739/htt...   \n",
       "368  https://web.archive.org/web/20090910073834/htt...   \n",
       "369  https://web.archive.org/web/20090919163055/htt...   \n",
       "370  http://transcripts.cnn.com/TRANSCRIPTS/0804/13...   \n",
       "371  https://web.archive.org/web/20070808173432/htt...   \n",
       "372  http://thomas.loc.gov/cgi-bin/query/D?c108:5:....   \n",
       "373  http://abcnews.go.com/ThisWeek/week-transcript...   \n",
       "374  http://www.colbertnation.com/the-colbert-repor...   \n",
       "375  http://www.politifact.com/truth-o-meter/statem...   \n",
       "376  https://web.archive.org/web/20150816005009/htt...   \n",
       "377  http://www.rawstory.com/rs/2012/07/17/sen-sand...   \n",
       "378  http://www.politicsdaily.com/2011/02/06/transc...   \n",
       "379  http://www.nytimes.com/2008/02/26/us/politics/...   \n",
       "380  http://www.politifact.com/texas/statements/201...   \n",
       "381  https://www.washingtonpost.com/news/the-fix/wp...   \n",
       "382  http://www.cq.com/doc/newsmakertranscripts-395...   \n",
       "383  http://www.thedailyshow.com/full-episodes/mon-...   \n",
       "384  http://select.nytimes.com/gst/abstract.html?re...   \n",
       "385  http://www.politifact.com/truth-o-meter/promis...   \n",
       "386  http://www.cnn.com/2017/01/05/politics/border-...   \n",
       "387         http://www.youtube.com/watch?v=4O8CxZ1OD58   \n",
       "388         http://www.youtube.com/watch?v=EhyMplwY6HY   \n",
       "389  https://web.archive.org/web/20071102131244/htt...   \n",
       "\n",
       "                                             top_img  \n",
       "0    {'Top_img': 'TNvleUERVcteFhXLFMOhJNNKtVJ7Gvtj'}  \n",
       "1    {'Top_img': 'F5EvpyGbVWnmGxY51SFwiwFApug6uCQD'}  \n",
       "2    {'Top_img': 'DYJ719DYWZuuhTiYC24sKe8j4xtXAmSS'}  \n",
       "3    {'Top_img': 'nhnAUdrFS91vNVbtAC9GJHYkCcJRJEWl'}  \n",
       "4    {'Top_img': 'B8gcMTZ9WskrcKCyVXtLr1df9gUbArSB'}  \n",
       "5    {'Top_img': 'wV4ZL7x32sUA3ywBfOj4bqt1FK7modqs'}  \n",
       "6    {'Top_img': 'y7ZQMDtegVuKlqLfU0fDRjzaeVOT9mf8'}  \n",
       "7    {'Top_img': 'GBv7hlj158k63w8HmPEJRuESLwqxgFrv'}  \n",
       "8    {'Top_img': '9HDdEGQ17SAzBEOlvzDnrzd8OdtzczjB'}  \n",
       "9    {'Top_img': 'SQ3WBpBmtpwTv1xWkaAwHIq53s6mYnUP'}  \n",
       "10   {'Top_img': 'qLXWHatMmE4ewhKbklBbknHS78Dp55kW'}  \n",
       "11   {'Top_img': 'NCi9wPcK0ANwMx4xBTFnjMiKDAxElLbk'}  \n",
       "12   {'Top_img': 'W00QKgIsasu1zh6U7d2YHPAYxQgZIZ2w'}  \n",
       "13   {'Top_img': 'U8CdJ7HvSZQPQoJKp80RkzScl2Gch3rW'}  \n",
       "14   {'Top_img': 'VHcOJvN9ja63B7Q5Kmw7jTx8u4Em4Ppi'}  \n",
       "15   {'Top_img': 'iKZ9ap7dK36FGarboQzqLsE9PA4c5dmu'}  \n",
       "16   {'Top_img': '3InzvZMnmLXAWFnX4cLmaZIxQKDpploa'}  \n",
       "17   {'Top_img': 'EB78w48u8ncBBruU8fjHG5kPu13v0fnc'}  \n",
       "18                                                {}  \n",
       "19   {'Top_img': '5QktBFq3fpnIjSje5Jd0KlcTObBw6kKM'}  \n",
       "20   {'Top_img': 'MifJVwUOxoXl1wFgiXMdmlE6FRFtz556'}  \n",
       "21   {'Top_img': 'oGAiJSDBU9SkvMpqnlruS6tBpbKTmC1Q'}  \n",
       "22   {'Top_img': 'fWH0R2f8ok8mzjMfvNRE6yQuTwnHyO8f'}  \n",
       "23   {'Top_img': 'vK7Vu5soPM14EXQFWfChkcMYw50fcMf8'}  \n",
       "24   {'Top_img': 'L6rrNgReN4Jv0HCNimFw2SmUdgVp1w44'}  \n",
       "25   {'Top_img': 'HZrLmuOmu3vGOEAaYY16I9nU4usE2e5c'}  \n",
       "26   {'Top_img': 'zZRnNcFnG8mE98pdximr9XH3vuoBZfNT'}  \n",
       "27   {'Top_img': '6VLIS8SI9oAKlwOaY3D8vNbE2JMSBh7R'}  \n",
       "28   {'Top_img': 'dBzWyRQmjkC5fLXVkU5puIVEvWsWZE2H'}  \n",
       "29   {'Top_img': 'X5QQEWWU42DZ4z6BIYuHi0ZLAmnCYiTY'}  \n",
       "..                                               ...  \n",
       "360  {'Top_img': 'UkhSiFbFRW7HHGncbZV4bpHumSUnYJzM'}  \n",
       "361  {'Top_img': 'DtTdzqOZ0zSZ8NUsvTFWvaIJC8NOvZyj'}  \n",
       "362  {'Top_img': 'xQm9QeN9olDeAll4cdOdOBLOxwgLTES8'}  \n",
       "363  {'Top_img': 'qts3oDLkZ8K1AFrAzWGcmdgwkVQlADvG'}  \n",
       "364  {'Top_img': 'eSGHW8cCpuCeZCy4CfQxPG93UMwbF7Vj'}  \n",
       "365  {'Top_img': '6BsVhYlDnZXzzKULjNEhianebIGxuOPE'}  \n",
       "366  {'Top_img': 'EbFJqhoABxdaDJ7dI8PAYEeQVBtS9F6o'}  \n",
       "367  {'Top_img': 'kTsxjig8XvbrHhkjBbpmZKS0efizQYLD'}  \n",
       "368  {'Top_img': 'jFZOJrrMiRgooCJWv1YjxLT3ly8iIByI'}  \n",
       "369  {'Top_img': 'x9HCdWe79x27h181pAdTmTlf1Z5DTQxU'}  \n",
       "370  {'Top_img': 'BUekE6CUNKbTK8mt4I2JJFKqjxfA6i4E'}  \n",
       "371  {'Top_img': 'h6G3SGo4Q7E4rX7bC4eQarBaoWX50imd'}  \n",
       "372  {'Top_img': 'Rw7GCoj29yd6OINmpopXWFVTOXWmvIIL'}  \n",
       "373  {'Top_img': '0fNmYlMmlJgQlEMXCp10RIqo07gmqicM'}  \n",
       "374  {'Top_img': 'An8cnTMZNx4ju86WZeDwcvnXt6bSNrOA'}  \n",
       "375  {'Top_img': 'BONw3V1r0luiQ6bvp3knSzY4q1BNaqlT'}  \n",
       "376                                               {}  \n",
       "377  {'Top_img': 'JCiDofIqViLZSUQA9I0L0AdivnlwmjfJ'}  \n",
       "378  {'Top_img': '86VQIQFxqGFRVJjCifFMewDQVNteu0UO'}  \n",
       "379  {'Top_img': 'a0x9A12cmC8JWAXDRrTPs86xShCEU3vc'}  \n",
       "380  {'Top_img': 'qy8BQvHFDSgfl0dqgLur8XaewbNWJte9'}  \n",
       "381  {'Top_img': 'm8P3orzJ06kfF0jg2HmjfnF3jMSb8xjh'}  \n",
       "382  {'Top_img': 'WfwLXBWL3kESRdHB9ksEilmnZMcOUZtJ'}  \n",
       "383  {'Top_img': 'JevSKniuEt66nIoKszr2VC89JlRfE7MJ'}  \n",
       "384  {'Top_img': 'XIami3akSNyJzijgc6tf0HkMhmZmT7AT'}  \n",
       "385  {'Top_img': 'tjg7cSY2CUJ6NtDw3CfddbxSbiznjahC'}  \n",
       "386  {'Top_img': 'tAKcEzennyCleZiMbqBDhRMGgNQUIjLy'}  \n",
       "387  {'Top_img': 'vgTBvxWraHslIeqmQybPUhLPikP1rzp2'}  \n",
       "388  {'Top_img': '5W20wJ2n3eWxudEijJ5eCopF78lbAYIg'}  \n",
       "389  {'Top_img': 'Zy1yA9joK5UgSybPCJ2m6stszAq7NQvF'}  \n",
       "\n",
       "[390 rows x 2 columns]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "poli_top"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = (dataset[['url']].merge(poli_top, on='url', how='left'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>top_img</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://www.nfib-sbet.org/</td>\n",
       "      <td>{'Top_img': 'TNvleUERVcteFhXLFMOhJNNKtVJ7Gvtj'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://www.cq.com/doc/newsmakertranscripts-494...</td>\n",
       "      <td>{'Top_img': 'F5EvpyGbVWnmGxY51SFwiwFApug6uCQD'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://web.archive.org/web/20080204072132/htt...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://web.archive.org/web/20110811143753/htt...</td>\n",
       "      <td>{'Top_img': 'DYJ719DYWZuuhTiYC24sKe8j4xtXAmSS'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://web.archive.org/web/20070820164107/htt...</td>\n",
       "      <td>{'Top_img': 'nhnAUdrFS91vNVbtAC9GJHYkCcJRJEWl'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>http://www.politifact.com/truth-o-meter/statem...</td>\n",
       "      <td>{'Top_img': 'B8gcMTZ9WskrcKCyVXtLr1df9gUbArSB'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>https://www.law.cornell.edu/constitution/amend...</td>\n",
       "      <td>{'Top_img': 'wV4ZL7x32sUA3ywBfOj4bqt1FK7modqs'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>https://web.archive.org/web/20090913221204/htt...</td>\n",
       "      <td>{'Top_img': 'y7ZQMDtegVuKlqLfU0fDRjzaeVOT9mf8'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>https://web.archive.org/web/20080623122709/htt...</td>\n",
       "      <td>{'Top_img': 'GBv7hlj158k63w8HmPEJRuESLwqxgFrv'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>http://www.cbsnews.com/blogs/2010/02/23/politi...</td>\n",
       "      <td>{'Top_img': '9HDdEGQ17SAzBEOlvzDnrzd8OdtzczjB'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>http://politicaladarchive.org/ad/polad_donaldt...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>http://data.bls.gov/timeseries/CES0500000001?o...</td>\n",
       "      <td>{'Top_img': 'SQ3WBpBmtpwTv1xWkaAwHIq53s6mYnUP'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>http://www.kff.org/healthreform/upload/8061.pdf</td>\n",
       "      <td>{'Top_img': 'qLXWHatMmE4ewhKbklBbknHS78Dp55kW'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>http://abcnews.go.com/ThisWeek/week-transcript...</td>\n",
       "      <td>{'Top_img': 'NCi9wPcK0ANwMx4xBTFnjMiKDAxElLbk'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>https://youtu.be/EiZW41bVF9I</td>\n",
       "      <td>{'Top_img': 'W00QKgIsasu1zh6U7d2YHPAYxQgZIZ2w'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>https://web.archive.org/web/20080506120114/htt...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>https://www.youtube.com/watch?v=AtJWEAj1vZM</td>\n",
       "      <td>{'Top_img': 'U8CdJ7HvSZQPQoJKp80RkzScl2Gch3rW'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>http://www.politifact.com/truth-o-meter/statem...</td>\n",
       "      <td>{'Top_img': 'VHcOJvN9ja63B7Q5Kmw7jTx8u4Em4Ppi'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>http://www.epi.org/blog/lost-decade-working-ag...</td>\n",
       "      <td>{'Top_img': 'iKZ9ap7dK36FGarboQzqLsE9PA4c5dmu'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>http://www.senate.gov/pagelayout/reference/nom...</td>\n",
       "      <td>{'Top_img': '3InzvZMnmLXAWFnX4cLmaZIxQKDpploa'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>https://medium.com/@PolitiFact/politifacts-ann...</td>\n",
       "      <td>{'Top_img': 'EB78w48u8ncBBruU8fjHG5kPu13v0fnc'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>https://web.archive.org/web/20100523071827/htt...</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>http://abcnews.go.com/ThisWeek/week-transcript...</td>\n",
       "      <td>{'Top_img': '5QktBFq3fpnIjSje5Jd0KlcTObBw6kKM'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>https://web.archive.org/web/20161007194921/htt...</td>\n",
       "      <td>{'Top_img': 'MifJVwUOxoXl1wFgiXMdmlE6FRFtz556'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>http://www.senate.gov/legislative/LIS/roll_cal...</td>\n",
       "      <td>{'Top_img': 'oGAiJSDBU9SkvMpqnlruS6tBpbKTmC1Q'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>http://www.youtube.com/watch?v=wyoZJUKL3wA&amp;fea...</td>\n",
       "      <td>{'Top_img': 'fWH0R2f8ok8mzjMfvNRE6yQuTwnHyO8f'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>http://www.bls.gov/ces/</td>\n",
       "      <td>{'Top_img': 'vK7Vu5soPM14EXQFWfChkcMYw50fcMf8'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>http://www.nytimes.com/2011/07/11/opinion/11kr...</td>\n",
       "      <td>{'Top_img': 'L6rrNgReN4Jv0HCNimFw2SmUdgVp1w44'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>http://www.nytimes.com/2008/06/03/us/politics/...</td>\n",
       "      <td>{'Top_img': 'HZrLmuOmu3vGOEAaYY16I9nU4usE2e5c'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>http://www.politico.com/arena/perm/John_Lewis_...</td>\n",
       "      <td>{'Top_img': 'zZRnNcFnG8mE98pdximr9XH3vuoBZfNT'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>714</th>\n",
       "      <td>https://web.archive.org/web/20180605205653/htt...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>715</th>\n",
       "      <td>https://web.archive.org/web/20180424000839/htt...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>716</th>\n",
       "      <td>https://web.archive.org/web/20170626190612/htt...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>717</th>\n",
       "      <td>https://web.archive.org/web/20170211071325/htt...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718</th>\n",
       "      <td>https://www.archives.gov/presidential-librarie...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>719</th>\n",
       "      <td>http://conservative-headlines.com/2015/03/sout...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>720</th>\n",
       "      <td>https://washingtonpress.com/2018/06/25/a-trump...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>721</th>\n",
       "      <td>https://web.archive.org/web/20150214123436/htt...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>722</th>\n",
       "      <td>http://usanewstoday.org/index.php/2017/01/02/c...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>723</th>\n",
       "      <td>https://boingboing.net/2018/04/09/fox-news-acc...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>724</th>\n",
       "      <td>https://www.hollywoodreporter.com/live-feed/da...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>725</th>\n",
       "      <td>www.reddit.com/r/conspiracy/comments/7ww8j1/ru...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>726</th>\n",
       "      <td>http://www.fox5dc.com/news/local-news/25485233...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>727</th>\n",
       "      <td>https://newstarget.com/2016-02-23-usda-approve...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>728</th>\n",
       "      <td>http://www.rearfront.com/about-us/</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>729</th>\n",
       "      <td>https://www.duffelblog.com/2017/10/bowe-bergda...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>730</th>\n",
       "      <td>http://babylonbee.com/news/isis-lays-arms-katy...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>731</th>\n",
       "      <td>https://web.archive.org/web/20170809163245/htt...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>732</th>\n",
       "      <td>patrioticviralnews.com/articles/international-...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>733</th>\n",
       "      <td>https://www.apnews.com/2cce92805c2749c297d0bbc...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>734</th>\n",
       "      <td>https://www.facebook.com/StopDjTrump/photos/a....</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>735</th>\n",
       "      <td>empirenews.net/police-discover-meth-lab-in-bac...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736</th>\n",
       "      <td>http://www.neonnettle.com/features/1306-twitte...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>737</th>\n",
       "      <td>https://web.archive.org/web/20170528095037/htt...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>738</th>\n",
       "      <td>https://web.archive.org/web/20170602190500/htt...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>739</th>\n",
       "      <td>genius.com/discussions/219349-Donald-trump-pro...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>740</th>\n",
       "      <td>http://thehill.com/homenews/senate/369928-who-...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>741</th>\n",
       "      <td>https://web.archive.org/web/20170322070001/htt...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>742</th>\n",
       "      <td>http://archive.is/DQqCx</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>743</th>\n",
       "      <td>politicono.com/palin-calls-boycott-mall-americ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>744 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   url  \\\n",
       "0                            http://www.nfib-sbet.org/   \n",
       "1    http://www.cq.com/doc/newsmakertranscripts-494...   \n",
       "2    https://web.archive.org/web/20080204072132/htt...   \n",
       "3    https://web.archive.org/web/20110811143753/htt...   \n",
       "4    https://web.archive.org/web/20070820164107/htt...   \n",
       "5    http://www.politifact.com/truth-o-meter/statem...   \n",
       "6    https://www.law.cornell.edu/constitution/amend...   \n",
       "7    https://web.archive.org/web/20090913221204/htt...   \n",
       "8    https://web.archive.org/web/20080623122709/htt...   \n",
       "9    http://www.cbsnews.com/blogs/2010/02/23/politi...   \n",
       "10   http://politicaladarchive.org/ad/polad_donaldt...   \n",
       "11   http://data.bls.gov/timeseries/CES0500000001?o...   \n",
       "12     http://www.kff.org/healthreform/upload/8061.pdf   \n",
       "13   http://abcnews.go.com/ThisWeek/week-transcript...   \n",
       "14                        https://youtu.be/EiZW41bVF9I   \n",
       "15   https://web.archive.org/web/20080506120114/htt...   \n",
       "16         https://www.youtube.com/watch?v=AtJWEAj1vZM   \n",
       "17   http://www.politifact.com/truth-o-meter/statem...   \n",
       "18   http://www.epi.org/blog/lost-decade-working-ag...   \n",
       "19   http://www.senate.gov/pagelayout/reference/nom...   \n",
       "20   https://medium.com/@PolitiFact/politifacts-ann...   \n",
       "21   https://web.archive.org/web/20100523071827/htt...   \n",
       "22   http://abcnews.go.com/ThisWeek/week-transcript...   \n",
       "23   https://web.archive.org/web/20161007194921/htt...   \n",
       "24   http://www.senate.gov/legislative/LIS/roll_cal...   \n",
       "25   http://www.youtube.com/watch?v=wyoZJUKL3wA&fea...   \n",
       "26                             http://www.bls.gov/ces/   \n",
       "27   http://www.nytimes.com/2011/07/11/opinion/11kr...   \n",
       "28   http://www.nytimes.com/2008/06/03/us/politics/...   \n",
       "29   http://www.politico.com/arena/perm/John_Lewis_...   \n",
       "..                                                 ...   \n",
       "714  https://web.archive.org/web/20180605205653/htt...   \n",
       "715  https://web.archive.org/web/20180424000839/htt...   \n",
       "716  https://web.archive.org/web/20170626190612/htt...   \n",
       "717  https://web.archive.org/web/20170211071325/htt...   \n",
       "718  https://www.archives.gov/presidential-librarie...   \n",
       "719  http://conservative-headlines.com/2015/03/sout...   \n",
       "720  https://washingtonpress.com/2018/06/25/a-trump...   \n",
       "721  https://web.archive.org/web/20150214123436/htt...   \n",
       "722  http://usanewstoday.org/index.php/2017/01/02/c...   \n",
       "723  https://boingboing.net/2018/04/09/fox-news-acc...   \n",
       "724  https://www.hollywoodreporter.com/live-feed/da...   \n",
       "725  www.reddit.com/r/conspiracy/comments/7ww8j1/ru...   \n",
       "726  http://www.fox5dc.com/news/local-news/25485233...   \n",
       "727  https://newstarget.com/2016-02-23-usda-approve...   \n",
       "728                 http://www.rearfront.com/about-us/   \n",
       "729  https://www.duffelblog.com/2017/10/bowe-bergda...   \n",
       "730  http://babylonbee.com/news/isis-lays-arms-katy...   \n",
       "731  https://web.archive.org/web/20170809163245/htt...   \n",
       "732  patrioticviralnews.com/articles/international-...   \n",
       "733  https://www.apnews.com/2cce92805c2749c297d0bbc...   \n",
       "734  https://www.facebook.com/StopDjTrump/photos/a....   \n",
       "735  empirenews.net/police-discover-meth-lab-in-bac...   \n",
       "736  http://www.neonnettle.com/features/1306-twitte...   \n",
       "737  https://web.archive.org/web/20170528095037/htt...   \n",
       "738  https://web.archive.org/web/20170602190500/htt...   \n",
       "739  genius.com/discussions/219349-Donald-trump-pro...   \n",
       "740  http://thehill.com/homenews/senate/369928-who-...   \n",
       "741  https://web.archive.org/web/20170322070001/htt...   \n",
       "742                            http://archive.is/DQqCx   \n",
       "743  politicono.com/palin-calls-boycott-mall-americ...   \n",
       "\n",
       "                                             top_img  \n",
       "0    {'Top_img': 'TNvleUERVcteFhXLFMOhJNNKtVJ7Gvtj'}  \n",
       "1    {'Top_img': 'F5EvpyGbVWnmGxY51SFwiwFApug6uCQD'}  \n",
       "2                                                NaN  \n",
       "3    {'Top_img': 'DYJ719DYWZuuhTiYC24sKe8j4xtXAmSS'}  \n",
       "4    {'Top_img': 'nhnAUdrFS91vNVbtAC9GJHYkCcJRJEWl'}  \n",
       "5    {'Top_img': 'B8gcMTZ9WskrcKCyVXtLr1df9gUbArSB'}  \n",
       "6    {'Top_img': 'wV4ZL7x32sUA3ywBfOj4bqt1FK7modqs'}  \n",
       "7    {'Top_img': 'y7ZQMDtegVuKlqLfU0fDRjzaeVOT9mf8'}  \n",
       "8    {'Top_img': 'GBv7hlj158k63w8HmPEJRuESLwqxgFrv'}  \n",
       "9    {'Top_img': '9HDdEGQ17SAzBEOlvzDnrzd8OdtzczjB'}  \n",
       "10                                               NaN  \n",
       "11   {'Top_img': 'SQ3WBpBmtpwTv1xWkaAwHIq53s6mYnUP'}  \n",
       "12   {'Top_img': 'qLXWHatMmE4ewhKbklBbknHS78Dp55kW'}  \n",
       "13   {'Top_img': 'NCi9wPcK0ANwMx4xBTFnjMiKDAxElLbk'}  \n",
       "14   {'Top_img': 'W00QKgIsasu1zh6U7d2YHPAYxQgZIZ2w'}  \n",
       "15                                               NaN  \n",
       "16   {'Top_img': 'U8CdJ7HvSZQPQoJKp80RkzScl2Gch3rW'}  \n",
       "17   {'Top_img': 'VHcOJvN9ja63B7Q5Kmw7jTx8u4Em4Ppi'}  \n",
       "18   {'Top_img': 'iKZ9ap7dK36FGarboQzqLsE9PA4c5dmu'}  \n",
       "19   {'Top_img': '3InzvZMnmLXAWFnX4cLmaZIxQKDpploa'}  \n",
       "20   {'Top_img': 'EB78w48u8ncBBruU8fjHG5kPu13v0fnc'}  \n",
       "21                                                {}  \n",
       "22   {'Top_img': '5QktBFq3fpnIjSje5Jd0KlcTObBw6kKM'}  \n",
       "23   {'Top_img': 'MifJVwUOxoXl1wFgiXMdmlE6FRFtz556'}  \n",
       "24   {'Top_img': 'oGAiJSDBU9SkvMpqnlruS6tBpbKTmC1Q'}  \n",
       "25   {'Top_img': 'fWH0R2f8ok8mzjMfvNRE6yQuTwnHyO8f'}  \n",
       "26   {'Top_img': 'vK7Vu5soPM14EXQFWfChkcMYw50fcMf8'}  \n",
       "27   {'Top_img': 'L6rrNgReN4Jv0HCNimFw2SmUdgVp1w44'}  \n",
       "28   {'Top_img': 'HZrLmuOmu3vGOEAaYY16I9nU4usE2e5c'}  \n",
       "29   {'Top_img': 'zZRnNcFnG8mE98pdximr9XH3vuoBZfNT'}  \n",
       "..                                               ...  \n",
       "714                                              NaN  \n",
       "715                                              NaN  \n",
       "716                                              NaN  \n",
       "717                                              NaN  \n",
       "718                                              NaN  \n",
       "719                                              NaN  \n",
       "720                                              NaN  \n",
       "721                                              NaN  \n",
       "722                                              NaN  \n",
       "723                                              NaN  \n",
       "724                                              NaN  \n",
       "725                                              NaN  \n",
       "726                                              NaN  \n",
       "727                                              NaN  \n",
       "728                                              NaN  \n",
       "729                                              NaN  \n",
       "730                                              NaN  \n",
       "731                                              NaN  \n",
       "732                                              NaN  \n",
       "733                                              NaN  \n",
       "734                                              NaN  \n",
       "735                                              NaN  \n",
       "736                                              NaN  \n",
       "737                                              NaN  \n",
       "738                                              NaN  \n",
       "739                                              NaN  \n",
       "740                                              NaN  \n",
       "741                                              NaN  \n",
       "742                                              NaN  \n",
       "743                                              NaN  \n",
       "\n",
       "[744 rows x 2 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (BERT)",
   "language": "python",
   "name": "bert"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
